{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import layers , activations , models , preprocessing , utils\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "对话内容总数： 4384\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input</th>\n",
       "      <th>output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>早安</td>\n",
       "      <td>早安 昨天晚上 睡得 好 吗</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>睡得 很 好</td>\n",
       "      <td>真是 不错 那 赶快 去 享用 美味 的 早餐 吧</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>早安</td>\n",
       "      <td>早安 昨天晚上 睡得 好 吗</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>睡得 好 啊</td>\n",
       "      <td>真棒 需要 去 外面 走走 做 早晨 运动 吗</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>好 啊 感觉 不错</td>\n",
       "      <td>那 记得 不要 做 太 激烈 的 运动 唷</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       input                     output\n",
       "0         早安             早安 昨天晚上 睡得 好 吗\n",
       "1     睡得 很 好  真是 不错 那 赶快 去 享用 美味 的 早餐 吧\n",
       "2         早安             早安 昨天晚上 睡得 好 吗\n",
       "3     睡得 好 啊    真棒 需要 去 外面 走走 做 早晨 运动 吗\n",
       "4  好 啊 感觉 不错      那 记得 不要 做 太 激烈 的 运动 唷"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取处理过的文本\n",
    "text_Segment = open('./Source_segment11.txt','r', encoding='utf-8')\n",
    "text_Segment_list = text_Segment.readlines()\n",
    "text_Segment.close()\n",
    "# 移除换行\n",
    "text_Segment_list = [n.rstrip() for n in text_Segment_list]\n",
    "if len(text_Segment_list)%2!=0:\n",
    "    print(\"文本库数据有误 对话不对称 请检查！\")\n",
    "else:\n",
    "    print('对话内容总数：', len(text_Segment_list))\n",
    "\n",
    "X = text_Segment_list[0:][::2] # 输入问句\n",
    "Y = text_Segment_list[1:][::2] # 输出答句\n",
    "\n",
    "lines = pd.DataFrame({\"input\":X,\"output\":Y})\n",
    "lines.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据填充"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input max length is 20\n",
      "Encoder input data shape -> (2192, 20)\n",
      "Number of Input tokens = 1616\n",
      "Output max length is 132\n",
      "Decoder input data shape -> (2192, 132)\n",
      "Number of Input tokens = 3718\n",
      "Decoder target data shape -> (2192, 132, 3718)\n"
     ]
    }
   ],
   "source": [
    "#encoder\n",
    "input_lines = list()\n",
    "for line in lines.input:\n",
    "    input_lines.append(line)\n",
    "    \n",
    "tokenizer = preprocessing.text.Tokenizer()\n",
    "tokenizer.fit_on_texts(input_lines)\n",
    "tokenized_input_lines = tokenizer.texts_to_sequences(input_lines)\n",
    "\n",
    "len_list = list()\n",
    "for token_line in tokenized_input_lines:\n",
    "    len_list.append(len(token_line))\n",
    "max_len = np.array(len_list).max()\n",
    "print( 'Input max length is {}'.format( max_len ))\n",
    "\n",
    "padded_input_lines = preprocessing.sequence.pad_sequences(tokenized_input_lines, maxlen=max_len, padding='post')\n",
    "encoder_input_data = np.array(padded_input_lines)\n",
    "print( 'Encoder input data shape -> {}'.format( encoder_input_data.shape ))\n",
    "\n",
    "input_word_dict = tokenizer.word_index\n",
    "num_input_tokens = len(input_word_dict) + 1 \n",
    "print( 'Number of Input tokens = {}'.format( num_input_tokens))\n",
    "\n",
    "\n",
    "#decoder\n",
    "output_lines = list()\n",
    "for line in lines.output:\n",
    "    output_lines.append('<START> ' + line +  ' <END>')\n",
    "    \n",
    "tokenizer = preprocessing.text.Tokenizer()\n",
    "tokenizer.fit_on_texts(output_lines)\n",
    "tokenized_output_lines = tokenizer.texts_to_sequences(output_lines)\n",
    "\n",
    "length_list = list()\n",
    "for token_seq in tokenized_output_lines:\n",
    "    length_list.append( len( token_seq ))\n",
    "max_output_length = np.array( length_list ).max()\n",
    "print( 'Output max length is {}'.format( max_output_length ))\n",
    "\n",
    "padded_output_lines = preprocessing.sequence.pad_sequences(tokenized_output_lines, maxlen=max_output_length, padding='post')\n",
    "decoder_input_data = np.array(padded_output_lines)\n",
    "print( 'Decoder input data shape -> {}'.format( decoder_input_data.shape ))\n",
    "\n",
    "output_word_dict = tokenizer.word_index\n",
    "num_output_tokens = len(output_word_dict) + 1 \n",
    "print( 'Number of Input tokens = {}'.format( num_output_tokens))\n",
    "\n",
    "\n",
    "#target\n",
    "decoder_target_data = list()\n",
    "for token in tokenized_output_lines:\n",
    "    decoder_target_data.append(token[1:])\n",
    "    \n",
    "padded_output_lines = preprocessing.sequence.pad_sequences(decoder_target_data, maxlen=max_output_length, padding='post')\n",
    "onehot_output_lines = utils.to_categorical(padded_output_lines, num_output_tokens)\n",
    "decoder_target_data = np.array(onehot_output_lines)\n",
    "print( 'Decoder target data shape -> {}'.format( decoder_target_data.shape ))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 搭建模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " input_1 (InputLayer)           [(None, None)]       0           []                               \n",
      "                                                                                                  \n",
      " input_2 (InputLayer)           [(None, None)]       0           []                               \n",
      "                                                                                                  \n",
      " embedding (Embedding)          (None, None, 256)    413696      ['input_1[0][0]']                \n",
      "                                                                                                  \n",
      " embedding_1 (Embedding)        (None, None, 256)    951808      ['input_2[0][0]']                \n",
      "                                                                                                  \n",
      " lstm (LSTM)                    [(None, 256),        525312      ['embedding[0][0]']              \n",
      "                                 (None, 256),                                                     \n",
      "                                 (None, 256)]                                                     \n",
      "                                                                                                  \n",
      " lstm_1 (LSTM)                  [(None, None, 256),  525312      ['embedding_1[0][0]',            \n",
      "                                 (None, 256),                     'lstm[0][1]',                   \n",
      "                                 (None, 256)]                     'lstm[0][2]']                   \n",
      "                                                                                                  \n",
      " attention_layer (Attention)    (None, None, 256)    0           ['lstm_1[0][0]',                 \n",
      "                                                                  'lstm[0][0]']                   \n",
      "                                                                                                  \n",
      " concat_layer (Concatenate)     (None, None, 512)    0           ['lstm_1[0][0]',                 \n",
      "                                                                  'attention_layer[0][0]']        \n",
      "                                                                                                  \n",
      " dense (Dense)                  (None, None, 3718)   1907334     ['concat_layer[0][0]']           \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 4,323,462\n",
      "Trainable params: 4,323,462\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "encoder_inputs = tf.keras.layers.Input(shape=( None , ))\n",
    "encoder_embedding = tf.keras.layers.Embedding( num_input_tokens, 256 , mask_zero=True ) (encoder_inputs)\n",
    "encoder_lstm = tf.keras.layers.LSTM( 256 , return_state=True , recurrent_activation = 'sigmoid',dropout=0.2)\n",
    "encoder_outputs , state_h , state_c = encoder_lstm( encoder_embedding )\n",
    "encoder_states = [ state_h , state_c ]\n",
    "\n",
    "decoder_inputs = tf.keras.layers.Input(shape=( None ,  ))\n",
    "decoder_embedding = tf.keras.layers.Embedding( num_output_tokens, 256 , mask_zero=True) (decoder_inputs)\n",
    "decoder_lstm = tf.keras.layers.LSTM( 256 , return_state=True , recurrent_activation = 'sigmoid',return_sequences=True,dropout=0.2)\n",
    "decoder_outputs , _ , _ = decoder_lstm ( decoder_embedding , initial_state=encoder_states)\n",
    "\n",
    "attention = tf.keras.layers.Attention(name='attention_layer')\n",
    "attention_output = attention([decoder_outputs,encoder_outputs])\n",
    "        \n",
    "decoder_concat = tf.keras.layers.Concatenate(axis=-1, name='concat_layer')\n",
    "decoder_concat_input = decoder_concat([decoder_outputs, attention_output])\n",
    "\n",
    "decoder_dense = tf.keras.layers.Dense( num_output_tokens , activation=tf.keras.activations.softmax ) \n",
    "output = decoder_dense ( decoder_concat_input )\n",
    "\n",
    "model = tf.keras.models.Model([encoder_inputs, decoder_inputs], output )\n",
    "model.compile(optimizer=tf.keras.optimizers.Adam(), loss='categorical_crossentropy',metrics=['accuracy'])\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "DEFAULT_BATCH_SIZE = 32\n",
    "DEFAULT_EPOCH = 200\n",
    "\n",
    "import random \n",
    "def generate_batch_data_random(x1,x2, y, batch_size):\n",
    "    \"\"\"逐步提取batch数据到显存，降低对显存的占用\"\"\"\n",
    "    ylen = len(y)\n",
    "    loopcount = ylen // batch_size\n",
    "    while (True):\n",
    "        i = random.randint(0,loopcount)\n",
    "        yield [x1[i * batch_size:(i + 1) * batch_size],x2[i * batch_size:(i + 1) * batch_size]], y[i * batch_size:(i + 1) * batch_size]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/200\n",
      "68/68 [==============================] - 10s 64ms/step - loss: 0.5759 - accuracy: 0.1220\n",
      "Epoch 2/200\n",
      "68/68 [==============================] - 4s 65ms/step - loss: 0.4832 - accuracy: 0.1746\n",
      "Epoch 3/200\n",
      "68/68 [==============================] - 4s 65ms/step - loss: 0.4394 - accuracy: 0.1937\n",
      "Epoch 4/200\n",
      "68/68 [==============================] - 4s 63ms/step - loss: 0.4267 - accuracy: 0.1913\n",
      "Epoch 5/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.3791 - accuracy: 0.2104\n",
      "Epoch 6/200\n",
      "68/68 [==============================] - 4s 63ms/step - loss: 0.3929 - accuracy: 0.2043\n",
      "Epoch 7/200\n",
      "68/68 [==============================] - 4s 66ms/step - loss: 0.3836 - accuracy: 0.2134\n",
      "Epoch 8/200\n",
      "68/68 [==============================] - 5s 66ms/step - loss: 0.3340 - accuracy: 0.2522\n",
      "Epoch 9/200\n",
      "68/68 [==============================] - 4s 63ms/step - loss: 0.3626 - accuracy: 0.2441\n",
      "Epoch 10/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.3252 - accuracy: 0.2974\n",
      "Epoch 11/200\n",
      "68/68 [==============================] - 4s 65ms/step - loss: 0.2667 - accuracy: 0.3617\n",
      "Epoch 12/200\n",
      "68/68 [==============================] - 4s 65ms/step - loss: 0.3032 - accuracy: 0.3362\n",
      "Epoch 13/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.2690 - accuracy: 0.3746\n",
      "Epoch 14/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.2620 - accuracy: 0.3831\n",
      "Epoch 15/200\n",
      "68/68 [==============================] - 4s 65ms/step - loss: 0.2372 - accuracy: 0.4110\n",
      "Epoch 16/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.2152 - accuracy: 0.4385\n",
      "Epoch 17/200\n",
      "68/68 [==============================] - 4s 65ms/step - loss: 0.2029 - accuracy: 0.5056\n",
      "Epoch 18/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.1994 - accuracy: 0.4942\n",
      "Epoch 19/200\n",
      "68/68 [==============================] - 4s 65ms/step - loss: 0.2003 - accuracy: 0.5013\n",
      "Epoch 20/200\n",
      "68/68 [==============================] - 4s 65ms/step - loss: 0.2119 - accuracy: 0.5022\n",
      "Epoch 21/200\n",
      "68/68 [==============================] - 4s 65ms/step - loss: 0.2010 - accuracy: 0.5213\n",
      "Epoch 22/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.1673 - accuracy: 0.5818\n",
      "Epoch 23/200\n",
      "68/68 [==============================] - 4s 65ms/step - loss: 0.1672 - accuracy: 0.5850\n",
      "Epoch 24/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.1676 - accuracy: 0.5766\n",
      "Epoch 25/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.1396 - accuracy: 0.6602\n",
      "Epoch 26/200\n",
      "68/68 [==============================] - 4s 65ms/step - loss: 0.1233 - accuracy: 0.6719\n",
      "Epoch 27/200\n",
      "68/68 [==============================] - 4s 63ms/step - loss: 0.1385 - accuracy: 0.6424\n",
      "Epoch 28/200\n",
      "68/68 [==============================] - 4s 63ms/step - loss: 0.1468 - accuracy: 0.6449\n",
      "Epoch 29/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.1213 - accuracy: 0.6830\n",
      "Epoch 30/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.1108 - accuracy: 0.7265\n",
      "Epoch 31/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0969 - accuracy: 0.7486\n",
      "Epoch 32/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.1045 - accuracy: 0.7364\n",
      "Epoch 33/200\n",
      "68/68 [==============================] - 4s 63ms/step - loss: 0.1081 - accuracy: 0.7248\n",
      "Epoch 34/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.1040 - accuracy: 0.7443\n",
      "Epoch 35/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0828 - accuracy: 0.7859\n",
      "Epoch 36/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0757 - accuracy: 0.8055\n",
      "Epoch 37/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0651 - accuracy: 0.8223\n",
      "Epoch 38/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0654 - accuracy: 0.8288\n",
      "Epoch 39/200\n",
      "68/68 [==============================] - 5s 68ms/step - loss: 0.0682 - accuracy: 0.8396\n",
      "Epoch 40/200\n",
      "68/68 [==============================] - 4s 66ms/step - loss: 0.0631 - accuracy: 0.8480\n",
      "Epoch 41/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.0668 - accuracy: 0.8357\n",
      "Epoch 42/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.0495 - accuracy: 0.8696\n",
      "Epoch 43/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0511 - accuracy: 0.8784\n",
      "Epoch 44/200\n",
      "68/68 [==============================] - 4s 63ms/step - loss: 0.0456 - accuracy: 0.8858\n",
      "Epoch 45/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0479 - accuracy: 0.8845\n",
      "Epoch 46/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0500 - accuracy: 0.8749\n",
      "Epoch 47/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0400 - accuracy: 0.8940\n",
      "Epoch 48/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0416 - accuracy: 0.8943\n",
      "Epoch 49/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0307 - accuracy: 0.9288\n",
      "Epoch 50/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0368 - accuracy: 0.9068\n",
      "Epoch 51/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0460 - accuracy: 0.8891\n",
      "Epoch 52/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0363 - accuracy: 0.9143\n",
      "Epoch 53/200\n",
      "68/68 [==============================] - 4s 65ms/step - loss: 0.0299 - accuracy: 0.9320\n",
      "Epoch 54/200\n",
      "68/68 [==============================] - 4s 65ms/step - loss: 0.0281 - accuracy: 0.9321\n",
      "Epoch 55/200\n",
      "68/68 [==============================] - 4s 66ms/step - loss: 0.0239 - accuracy: 0.9498\n",
      "Epoch 56/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0237 - accuracy: 0.9447\n",
      "Epoch 57/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0332 - accuracy: 0.9239\n",
      "Epoch 58/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.0278 - accuracy: 0.9309\n",
      "Epoch 59/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.0307 - accuracy: 0.9289\n",
      "Epoch 60/200\n",
      "68/68 [==============================] - 4s 63ms/step - loss: 0.0296 - accuracy: 0.9355\n",
      "Epoch 61/200\n",
      "68/68 [==============================] - 4s 63ms/step - loss: 0.0275 - accuracy: 0.9446\n",
      "Epoch 62/200\n",
      "68/68 [==============================] - 4s 63ms/step - loss: 0.0157 - accuracy: 0.9691\n",
      "Epoch 63/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0130 - accuracy: 0.9787\n",
      "Epoch 64/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0243 - accuracy: 0.9432\n",
      "Epoch 65/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0121 - accuracy: 0.9767\n",
      "Epoch 66/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0135 - accuracy: 0.9721\n",
      "Epoch 67/200\n",
      "68/68 [==============================] - 4s 63ms/step - loss: 0.0144 - accuracy: 0.9691\n",
      "Epoch 68/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.0153 - accuracy: 0.9678\n",
      "Epoch 69/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.0085 - accuracy: 0.9844\n",
      "Epoch 70/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0197 - accuracy: 0.9514\n",
      "Epoch 71/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0179 - accuracy: 0.9614\n",
      "Epoch 72/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0165 - accuracy: 0.9623\n",
      "Epoch 73/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0107 - accuracy: 0.9746\n",
      "Epoch 74/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0153 - accuracy: 0.9660\n",
      "Epoch 75/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0110 - accuracy: 0.9759\n",
      "Epoch 76/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0118 - accuracy: 0.9709\n",
      "Epoch 77/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0092 - accuracy: 0.9798\n",
      "Epoch 78/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0170 - accuracy: 0.9603\n",
      "Epoch 79/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0080 - accuracy: 0.9851\n",
      "Epoch 80/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0105 - accuracy: 0.9763\n",
      "Epoch 81/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.0090 - accuracy: 0.9781\n",
      "Epoch 82/200\n",
      "68/68 [==============================] - 4s 65ms/step - loss: 0.0072 - accuracy: 0.9850\n",
      "Epoch 83/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.0098 - accuracy: 0.9753\n",
      "Epoch 84/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0083 - accuracy: 0.9793\n",
      "Epoch 85/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0068 - accuracy: 0.9811\n",
      "Epoch 86/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0053 - accuracy: 0.9855\n",
      "Epoch 87/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0097 - accuracy: 0.9752\n",
      "Epoch 88/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0113 - accuracy: 0.9737\n",
      "Epoch 89/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0063 - accuracy: 0.9828\n",
      "Epoch 90/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0083 - accuracy: 0.9766\n",
      "Epoch 91/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0075 - accuracy: 0.9812\n",
      "Epoch 92/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0096 - accuracy: 0.9734\n",
      "Epoch 93/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0087 - accuracy: 0.9775\n",
      "Epoch 94/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0077 - accuracy: 0.9814\n",
      "Epoch 95/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0088 - accuracy: 0.9743\n",
      "Epoch 96/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.0075 - accuracy: 0.9819\n",
      "Epoch 97/200\n",
      "68/68 [==============================] - 4s 65ms/step - loss: 0.0048 - accuracy: 0.9851\n",
      "Epoch 98/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0075 - accuracy: 0.9814\n",
      "Epoch 99/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0090 - accuracy: 0.9791\n",
      "Epoch 100/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0065 - accuracy: 0.9818\n",
      "Epoch 101/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0072 - accuracy: 0.9807\n",
      "Epoch 102/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0080 - accuracy: 0.9758\n",
      "Epoch 103/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0090 - accuracy: 0.9712\n",
      "Epoch 104/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0081 - accuracy: 0.9790\n",
      "Epoch 105/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0055 - accuracy: 0.9826\n",
      "Epoch 106/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0062 - accuracy: 0.9816\n",
      "Epoch 107/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0071 - accuracy: 0.9828\n",
      "Epoch 108/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0053 - accuracy: 0.9827\n",
      "Epoch 109/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0043 - accuracy: 0.9885\n",
      "Epoch 110/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.0058 - accuracy: 0.9866\n",
      "Epoch 111/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.0052 - accuracy: 0.9863\n",
      "Epoch 112/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0048 - accuracy: 0.9870\n",
      "Epoch 113/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0066 - accuracy: 0.9792\n",
      "Epoch 114/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0070 - accuracy: 0.9785\n",
      "Epoch 115/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0043 - accuracy: 0.9862\n",
      "Epoch 116/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0041 - accuracy: 0.9879\n",
      "Epoch 117/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0053 - accuracy: 0.9855\n",
      "Epoch 118/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0044 - accuracy: 0.9866\n",
      "Epoch 119/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0054 - accuracy: 0.9835\n",
      "Epoch 120/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0043 - accuracy: 0.9867\n",
      "Epoch 121/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0048 - accuracy: 0.9862\n",
      "Epoch 122/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0058 - accuracy: 0.9816\n",
      "Epoch 123/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0039 - accuracy: 0.9875\n",
      "Epoch 124/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0032 - accuracy: 0.9882\n",
      "Epoch 125/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0048 - accuracy: 0.9807\n",
      "Epoch 126/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0042 - accuracy: 0.9837\n",
      "Epoch 127/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0051 - accuracy: 0.9808\n",
      "Epoch 128/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0034 - accuracy: 0.9882\n",
      "Epoch 129/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0040 - accuracy: 0.9867\n",
      "Epoch 130/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0051 - accuracy: 0.9819\n",
      "Epoch 131/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0043 - accuracy: 0.9860\n",
      "Epoch 132/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0047 - accuracy: 0.9822\n",
      "Epoch 133/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0049 - accuracy: 0.9801\n",
      "Epoch 134/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0059 - accuracy: 0.9823\n",
      "Epoch 135/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.0085 - accuracy: 0.9717\n",
      "Epoch 136/200\n",
      "68/68 [==============================] - 4s 63ms/step - loss: 0.0087 - accuracy: 0.9730\n",
      "Epoch 137/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0078 - accuracy: 0.9783\n",
      "Epoch 138/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0090 - accuracy: 0.9741\n",
      "Epoch 139/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0096 - accuracy: 0.9699\n",
      "Epoch 140/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0046 - accuracy: 0.9848\n",
      "Epoch 141/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0043 - accuracy: 0.9839\n",
      "Epoch 142/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0042 - accuracy: 0.9836\n",
      "Epoch 143/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0044 - accuracy: 0.9832\n",
      "Epoch 144/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0035 - accuracy: 0.9869\n",
      "Epoch 145/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0031 - accuracy: 0.9871\n",
      "Epoch 146/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0056 - accuracy: 0.9784\n",
      "Epoch 147/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0033 - accuracy: 0.9868\n",
      "Epoch 148/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0038 - accuracy: 0.9843\n",
      "Epoch 149/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0037 - accuracy: 0.9877\n",
      "Epoch 150/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0031 - accuracy: 0.9861\n",
      "Epoch 151/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0039 - accuracy: 0.9827\n",
      "Epoch 152/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0067 - accuracy: 0.9738\n",
      "Epoch 153/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0027 - accuracy: 0.9900\n",
      "Epoch 154/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0019 - accuracy: 0.9913\n",
      "Epoch 155/200\n",
      "68/68 [==============================] - 4s 66ms/step - loss: 0.0028 - accuracy: 0.9867\n",
      "Epoch 156/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0039 - accuracy: 0.9831\n",
      "Epoch 157/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0037 - accuracy: 0.9858\n",
      "Epoch 158/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0038 - accuracy: 0.9846\n",
      "Epoch 159/200\n",
      "68/68 [==============================] - 5s 68ms/step - loss: 0.0029 - accuracy: 0.9893\n",
      "Epoch 160/200\n",
      "68/68 [==============================] - 5s 78ms/step - loss: 0.0031 - accuracy: 0.9853\n",
      "Epoch 161/200\n",
      "68/68 [==============================] - 6s 84ms/step - loss: 0.0050 - accuracy: 0.9786\n",
      "Epoch 162/200\n",
      "68/68 [==============================] - 4s 66ms/step - loss: 0.0033 - accuracy: 0.9862\n",
      "Epoch 163/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0051 - accuracy: 0.9817\n",
      "Epoch 164/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0035 - accuracy: 0.9868\n",
      "Epoch 165/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.0035 - accuracy: 0.9853\n",
      "Epoch 166/200\n",
      "68/68 [==============================] - 4s 66ms/step - loss: 0.0036 - accuracy: 0.9841\n",
      "Epoch 167/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.0029 - accuracy: 0.9884\n",
      "Epoch 168/200\n",
      "68/68 [==============================] - 4s 63ms/step - loss: 0.0035 - accuracy: 0.9855\n",
      "Epoch 169/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0031 - accuracy: 0.9871\n",
      "Epoch 170/200\n",
      "68/68 [==============================] - 4s 63ms/step - loss: 0.0039 - accuracy: 0.9846\n",
      "Epoch 171/200\n",
      "68/68 [==============================] - 4s 59ms/step - loss: 0.0045 - accuracy: 0.9814\n",
      "Epoch 172/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0056 - accuracy: 0.9757\n",
      "Epoch 173/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0034 - accuracy: 0.9853\n",
      "Epoch 174/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0030 - accuracy: 0.9857\n",
      "Epoch 175/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.0022 - accuracy: 0.9919\n",
      "Epoch 176/200\n",
      "68/68 [==============================] - 5s 70ms/step - loss: 0.0031 - accuracy: 0.9859\n",
      "Epoch 177/200\n",
      "68/68 [==============================] - 5s 69ms/step - loss: 0.0029 - accuracy: 0.9877\n",
      "Epoch 178/200\n",
      "68/68 [==============================] - 5s 71ms/step - loss: 0.0023 - accuracy: 0.9903\n",
      "Epoch 179/200\n",
      "68/68 [==============================] - 5s 69ms/step - loss: 0.0035 - accuracy: 0.9838\n",
      "Epoch 180/200\n",
      "68/68 [==============================] - 4s 63ms/step - loss: 0.0031 - accuracy: 0.9897\n",
      "Epoch 181/200\n",
      "68/68 [==============================] - 4s 59ms/step - loss: 0.0049 - accuracy: 0.9810\n",
      "Epoch 182/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0061 - accuracy: 0.9783\n",
      "Epoch 183/200\n",
      "68/68 [==============================] - 4s 59ms/step - loss: 0.0073 - accuracy: 0.9717\n",
      "Epoch 184/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.0043 - accuracy: 0.9836\n",
      "Epoch 185/200\n",
      "68/68 [==============================] - 5s 67ms/step - loss: 0.0063 - accuracy: 0.9750\n",
      "Epoch 186/200\n",
      "68/68 [==============================] - 4s 64ms/step - loss: 0.0038 - accuracy: 0.9870\n",
      "Epoch 187/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0061 - accuracy: 0.9800\n",
      "Epoch 188/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0024 - accuracy: 0.9907\n",
      "Epoch 189/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0047 - accuracy: 0.9798\n",
      "Epoch 190/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0036 - accuracy: 0.9860\n",
      "Epoch 191/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0030 - accuracy: 0.9875\n",
      "Epoch 192/200\n",
      "68/68 [==============================] - 4s 60ms/step - loss: 0.0034 - accuracy: 0.9872\n",
      "Epoch 193/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0023 - accuracy: 0.9905\n",
      "Epoch 194/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0030 - accuracy: 0.9872\n",
      "Epoch 195/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0030 - accuracy: 0.9870\n",
      "Epoch 196/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0027 - accuracy: 0.9885\n",
      "Epoch 197/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0034 - accuracy: 0.9856\n",
      "Epoch 198/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0021 - accuracy: 0.9908\n",
      "Epoch 199/200\n",
      "68/68 [==============================] - 4s 62ms/step - loss: 0.0035 - accuracy: 0.9835\n",
      "Epoch 200/200\n",
      "68/68 [==============================] - 4s 61ms/step - loss: 0.0025 - accuracy: 0.9894\n"
     ]
    }
   ],
   "source": [
    "train_num_batches = len(encoder_input_data) // DEFAULT_BATCH_SIZE\n",
    "model.fit(generate_batch_data_random(encoder_input_data,decoder_input_data,decoder_target_data,DEFAULT_BATCH_SIZE)\n",
    ",steps_per_epoch=train_num_batches, batch_size=DEFAULT_BATCH_SIZE, epochs=DEFAULT_EPOCH) \n",
    "model.save( 'model.h5' ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model.fit([encoder_input_data , decoder_input_data], decoder_target_data, batch_size=64, epochs=250) \n",
    "#model.save( 'model.h5' ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_inference_model():\n",
    "    encoder_outputs , state_h , state_c = encoder_lstm( encoder_embedding )\n",
    "    encoder_model = tf.keras.models.Model(encoder_inputs, [encoder_outputs,encoder_states])\n",
    "    decoder_state_input_h = tf.keras.layers.Input(shape=(256,))\n",
    "    decoder_state_input_c = tf.keras.layers.Input(shape=(256,))\n",
    "    \n",
    "    decoder_state_inputs = [decoder_state_input_h, decoder_state_input_c]\n",
    "    \n",
    "    decoder_outputs, state_h, state_c = decoder_lstm(decoder_embedding, decoder_state_inputs)\n",
    "    decoder_states = [state_h, state_c]\n",
    "\n",
    "    attention_output = attention([decoder_outputs,encoder_outputs])\n",
    "    decoder_concat_input = decoder_concat([decoder_outputs, attention_output])\n",
    "    decoder_outputs = decoder_dense(decoder_concat_input)\n",
    "\n",
    "    decoder_model = tf.keras.models.Model([decoder_inputs,encoder_outputs] + decoder_state_inputs, [decoder_outputs] + decoder_states)\n",
    "    return encoder_model, decoder_model\n",
    "enc_model , dec_model = make_inference_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import jieba\n",
    "def str_to_token (sentence: str):\n",
    "    words = sentence.lower().strip()\n",
    "    words = jieba.cut(words)\n",
    "    token_list = list()\n",
    "    for word in words: \n",
    "        token_list.append(input_word_dict[word])\n",
    "    return preprocessing.sequence.pad_sequences([token_list], maxlen=max_len, padding='post')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache C:\\Users\\taki\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 0.723 seconds.\n",
      "Prefix dict has been built successfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "你好\n",
      "Bot:非常感谢你\n",
      "\n",
      "你\n",
      "在\n",
      "干嘛\n",
      "Bot:对不起，我没有听懂。\n",
      "\n",
      "我\n",
      "呸\n",
      "Bot:对不起，我没有听懂。\n",
      "\n",
      "hi\n",
      "Bot:对不起，我没有听懂。\n",
      "\n",
      "hello\n",
      "Bot:对不起，我没有听懂。\n",
      "\n",
      "我\n",
      "想\n",
      "你\n",
      "Bot:你当你穿越一个bug相对你会得到什么\n",
      "\n",
      "ai\n",
      "是\n",
      "什么\n",
      "Bot:历史：政治经济军事事件随着时间的推移和人的黎明ai时代的进程\n",
      "\n",
      "天气\n",
      "怎么样\n",
      "Bot:对不起，我没有听懂。\n",
      "\n",
      "天气\n",
      "好\n",
      "热\n",
      "Bot:多补充水分和多休息吧\n",
      "\n",
      "天气\n",
      "不好\n",
      "Bot:有favoritestoryis2001\n",
      "\n",
      "我爱你\n",
      "Bot:我也爱你feel机器人\n",
      "\n",
      "我爱你\n",
      "Bot:我也爱你feel机器人\n",
      "\n",
      "Bot:对不起，我没有听懂。\n",
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "model.load_weights(\"./model.h5\")\n",
    "\n",
    "for epoch in range( encoder_input_data.shape[0] ):\n",
    "    decoded_translation = ''\n",
    "    try:\n",
    "        encoder_outputs,states_values = enc_model.predict( str_to_token( input( 'User: ' ) ) )\n",
    "        empty_target_seq = np.zeros( ( 1 , 1 ) )\n",
    "        empty_target_seq[0, 0] = output_word_dict['start']\n",
    "        stop_condition = False\n",
    "        while not stop_condition :\n",
    "            dec_outputs , h , c = dec_model.predict([empty_target_seq,encoder_outputs] + states_values )\n",
    "            sampled_word_index = np.argmax( dec_outputs[0, -1, :] )\n",
    "            sampled_word = None\n",
    "            for word , index in output_word_dict.items() :\n",
    "                if sampled_word_index == index :\n",
    "                    decoded_translation += ' {}'.format( word )\n",
    "                    sampled_word = word\n",
    "            \n",
    "            if sampled_word == 'end' or len(decoded_translation.split()) > max_output_length:\n",
    "                stop_condition = True\n",
    "                \n",
    "            empty_target_seq = np.zeros( ( 1 , 1 ) )  \n",
    "            empty_target_seq[ 0 , 0 ] = sampled_word_index\n",
    "            states_values = [ h , c ] \n",
    "    except:\n",
    "        decoded_translation = '对不起，我没有听懂。'\n",
    "    print( \"Bot:\" +decoded_translation.replace(' end', '').replace(\" \",\"\"))\n",
    "    print()"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "ad2bdc8ecc057115af97d19610ffacc2b4e99fae6737bb82f5d7fb13d2f2c186"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
